#!/usr/local/bin/python
# -*- coding: utf8 -*-

import sys
import os

from BeautifulSoup import *

import mmseg
from mmseg.search import seg_txt_search,seg_txt_2_dict

#reload(sys)
#sys.setdefaultencoding('utf8') 

class ArticleIndexer:
  def GetContent(self, aid):
    f = open("articles-body/" + str(aid) + ".txt")
    res = ''
    lines = f.readlines()
    for l in lines:
      res += self.GetTextOnly(BeautifulSoup(l))

    return str(res)

  def GetTextOnly(self, soup):
    SPACE = '&nbsp;'
    v = soup.string
    if v == None:
      c = soup.contents
      result = ''
      for t in c:
        subtext = self.GetTextOnly(t)
        if subtext[len(subtext) - len(SPACE):] == SPACE:
          subtext = subtext[0 : len(subtext) - len(SPACE)]

        if len(result) > 0 and result[-1] != ' ':
          result += ' '
        result += subtext 
      return result
    else:
      # 过滤无信息的标签
      if v.parent.name in ('script', 'noscript', 'style'):
        return ''
      return v.strip()

  def MakeIndex(self, aid, title):
    text = self.GetContent(aid)
    print "----", aid, title
    features = {}

    for long_word in mmseg.seg_txt(text):
      if len(long_word.strip()) == 0:
        continue

      if long_word in features:
        features[long_word] += 1.0
      else:
        features[long_word] = 1.0 

      for word in seg_txt_search(long_word):
        if word == long_word:
          continue
        if word in features:
          features[word] += 1.0 
        else:
          features[word] = 1.0 

    base_length = len(text)
    if base_length < 1000:
      base_length = 1000
    
    for (k, count) in features.items():
      v = 10000 * (float(len(k)) * float(count)) / float(base_length)
      features[k] = v
    return features

  def BuildAllIndex(self):
    all_index = {}
    article_titles = open("x-article-titles.txt")
    count = 0

    for line in article_titles:
      pos = line.find('\t', 0)
      aid = int(line[:pos])
      title = line[pos + 1:]
      
      idx_file_name = "search-index/" + str(aid) + ".idx"
      
      if os.path.isfile(idx_file_name):
        print "pass file ", idx_file_name
        continue

      print "build file ", idx_file_name
      # print aid, title
      count += 1
      sys.stderr.write("article %s indexing %d ...\r\n" % (aid, count))
      article_words = indexer.MakeIndex(aid, title)
      print len(article_words)

      current_index_file = open(idx_file_name, "w")
      for (word, weight) in article_words.items():
        current_index_file.write("%d %s %f\r\n" % (aid, word, weight))
      current_index_file.close()
    article_titles.close()
    return all_index

if __name__ == '__main__':
  indexer = ArticleIndexer()

  m = indexer.BuildAllIndex()
  for (word, index) in m.items():
    print word, 
    for (weight, ids) in index.items():
      print weight, 
      for i in ids:
        print i,
      print ";",
    print ""

